DATA 622 Final Project - Image Classification

Setup

library(skimr)
library(tidyverse)
library(readr)
library(dplyr)
library(caret)
library(naivebayes)
library(factoextra) # For PCA plots
library(e1071)
library(Rtsne)
library(RColorBrewer)

Data

mnist_raw <- read_csv("https://pjreddie.com/media/files/mnist_train.csv", col_names = FALSE)
mnist_raw_test <- read_csv("https://pjreddie.com/media/files/mnist_test.csv", col_names = FALSE)

The MNIST dataset is a compilation of handwritten digits which have been digitzed for use in supervised machine learning classification applications. The datset is fairly large with 70,000 observations where each observation is a handwritten digit from various subjects. As a result of this diversity, each handwritten digit of the same number can have differences due to penmanship style and variation within the same penmanship style.

Each handwritten digit is on a 28x28 pixel image. The digits per observation range from 0 to 9 where each observation/digit in the 28x28 pixel have had their size normalized and have been centered on the image canvas.

Below is a sample of the number 3 from one of the observations in the dataset. Note that the images are grayscale

m <- t(matrix(mnist_raw[8,] %>% select(-X1), ncol = 28))
m2 <- matrix(unlist(m), nrow = 28, byrow = FALSE)
dimnames(m) <-list(rep("", dim(m)[1]), rep("", dim(m)[2]))

rotate <- function(x) t(apply(x, 2, rev))
image(rotate(m2), col=gray((255:0)/255))

Each every pixel on the canvas is represented by a integer range from 0 to 255, where 0 means the pixel is completely white and 255 means the pixel is completely black, the ranges from 1 to 254 are the various shades of the color gray. Since each image is 28x28 pixels in size, then each image can be represented by a 28x28 size matrix. Below is an a matrix representation of the number 3 (observation 8)

cat("Label: ", mnist_raw$X1[8], "\n")

## Label:  3

m

##                                                                               
##  0 0 0 0 0 0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0  
##  0 0 0 0 0 0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0  
##  0 0 0 0 0 0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0  
##  0 0 0 0 0 0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0  
##  0 0 0 0 0 0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0  
##  0 0 0 0 0 0   0   0   0   0   0   38  43  105 255 253 253 253 253 253 174 6  
##  0 0 0 0 0 0   0   0   0   43  139 224 226 252 253 252 252 252 252 252 252 158
##  0 0 0 0 0 0   0   0   0   178 252 252 252 252 253 252 252 252 252 252 252 252
##  0 0 0 0 0 0   0   0   0   109 252 252 230 132 133 132 132 189 252 252 252 252
##  0 0 0 0 0 0   0   0   0   4   29  29  24  0   0   0   0   14  226 252 252 172
##  0 0 0 0 0 0   0   0   0   0   0   0   0   0   0   0   0   85  243 252 252 144
##  0 0 0 0 0 0   0   0   0   0   0   0   0   0   0   0   88  189 252 252 252 14 
##  0 0 0 0 0 0   0   0   0   0   0   0   0   0   91  212 247 252 252 252 204 9  
##  0 0 0 0 0 0   0   0   0   32  125 193 193 193 253 252 252 252 238 102 28  0  
##  0 0 0 0 0 0   0   0   45  222 252 252 252 252 253 252 252 252 177 0   0   0  
##  0 0 0 0 0 0   0   0   45  223 253 253 253 253 255 253 253 253 253 74  0   0  
##  0 0 0 0 0 0   0   0   0   31  123 52  44  44  44  44  143 252 252 74  0   0  
##  0 0 0 0 0 0   0   0   0   0   0   0   0   0   0   0   15  252 252 74  0   0  
##  0 0 0 0 0 0   0   0   0   0   0   0   0   0   0   0   86  252 252 74  0   0  
##  0 0 0 0 0 0   5   75  9   0   0   0   0   0   0   98  242 252 252 74  0   0  
##  0 0 0 0 0 61  183 252 29  0   0   0   0   18  92  239 252 252 243 65  0   0  
##  0 0 0 0 0 208 252 252 147 134 134 134 134 203 253 252 252 188 83  0   0   0  
##  0 0 0 0 0 208 252 252 252 252 252 252 252 252 253 230 153 8   0   0   0   0  
##  0 0 0 0 0 49  157 252 252 252 252 252 217 207 146 45  0   0   0   0   0   0  
##  0 0 0 0 0 0   7   103 235 252 172 103 24  0   0   0   0   0   0   0   0   0  
##  0 0 0 0 0 0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0  
##  0 0 0 0 0 0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0  
##  0 0 0 0 0 0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0  
##              
##  0  0 0 0 0 0
##  0  0 0 0 0 0
##  0  0 0 0 0 0
##  0  0 0 0 0 0
##  0  0 0 0 0 0
##  0  0 0 0 0 0
##  14 0 0 0 0 0
##  59 0 0 0 0 0
##  59 0 0 0 0 0
##  7  0 0 0 0 0
##  0  0 0 0 0 0
##  0  0 0 0 0 0
##  0  0 0 0 0 0
##  0  0 0 0 0 0
##  0  0 0 0 0 0
##  0  0 0 0 0 0
##  0  0 0 0 0 0
##  0  0 0 0 0 0
##  0  0 0 0 0 0
##  0  0 0 0 0 0
##  0  0 0 0 0 0
##  0  0 0 0 0 0
##  0  0 0 0 0 0
##  0  0 0 0 0 0
##  0  0 0 0 0 0
##  0  0 0 0 0 0
##  0  0 0 0 0 0
##  0  0 0 0 0 0

To represent each matrix as an observation, each digit matrix has been flattened (converting from a multidimensional array to a single dimensional array) such that each observation is a integer list of (\(28x28=784\)) length 784, where each value in the list is the pixel value ranging from 0 to 255. This list has an additional value which contains a number from the range 0-9 containing the label value of the number represented by the image, bringing the length per observation to 785.

Bringing this all together, the dataset has 70,000 observations (60,000 training and 10,000 testing), 784 features and 1 classification label. The dataset of digital images is now represented mathematically that we can perform additional analysis and modeling on it.

# Reduce the dataset down from 60,000 observations
mnist_subset <- mnist_raw %>%  head(5000)
# Relabel X1 and add instance number
mnist_subset <- mnist_subset  %>%  rename(label = X1) %>%  mutate(instance = row_number())
# Gather columns into x, y values
mnist_subset 

X <- mnist_subset %>% select(contains('X'))
y <- mnist_subset$label 

pixels_gathered <- mnist_subset %>%  gather(pixel, value, -label, -instance) %>%  tidyr::extract(pixel, "pixel", "(\\d+)", convert = TRUE) %>%  mutate(pixel = pixel - 2, x = pixel %% 28, y = 28 - pixel %/% 28)
pixels_gathered

Example instances. Replace by 10x10 tile plot shoring 10 instance of 1 number for every row

theme_set(theme_light())
pixels_gathered %>%  filter(instance <=12) %>%  ggplot(aes(x, y, fill = value)) +  geom_tile() +  facet_wrap(~ instance + label) + scale_fill_gradient(low = "white", high = "black")

Data Exploration

Fairly even distribution. Can also try random sampling instead of head(10000) at the top.

ggplot(mnist_subset, aes(label)) + geom_bar(fill="steelblue")

The majority of points are either 0 (white) or 255 (black). Most values are not useful. Dimensionality reduction

ggplot(pixels_gathered, aes(value)) +  geom_histogram(bins=256, fill="steelblue")

pixel_summary <- pixels_gathered %>%  group_by(x, y, label) %>%  summarize(mean_value = mean(value)) %>%  ungroup()

## `summarise()` regrouping output by 'x', 'y' (override with `.groups` argument)

Average numbers representations. Gives a good idea of variability

pixel_summary %>%  ggplot(aes(x, y, fill = mean_value)) +  geom_tile() +  scale_fill_gradient2(low = "white", high = "black", mid = "gray", midpoint = 127.5) +  facet_wrap(~ label, nrow = 2) +  labs(title = "Average value of each pixel in 10 MNIST digits",       fill = "Average value") +  theme_void()

digit_differences <- crossing(compare1 = 0:9, compare2 = 0:9) %>%  filter(compare1 != compare2) %>%  mutate(negative = compare1, positive = compare2) %>%  gather(class, label, positive, negative) %>%  inner_join(pixel_summary, by = "label") %>%  select(-label) %>%  spread(class, mean_value)
ggplot(digit_differences, aes(x, y, fill = positive - negative)) +  geom_tile() +  scale_fill_gradient2(low = "blue", high = "red", mid = "white", midpoint = .5) +  facet_grid(compare2 ~ compare1) +  theme_void() +  labs(title = "Pixels that distinguish pairs of MNIST images",       subtitle = "Red means the pixel is darker for that row's digit, and blue means the pixel is darker for that column's digit.")

nonzero_var <- nearZeroVar(X, saveMetrics = FALSE)
image(matrix(1:784 %in% nonzero_var, 28, 28))

x <- 8
cat("Label: ", mnist_raw$X1[x], "\n")

## Label:  3

m <- t(matrix(mnist_raw[x,] %>% select(-X1), ncol = 28))
dimnames(m) <-list(rep("", dim(m)[1]), rep("", dim(m)[2]))
m

##                                                                               
##  0 0 0 0 0 0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0  
##  0 0 0 0 0 0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0  
##  0 0 0 0 0 0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0  
##  0 0 0 0 0 0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0  
##  0 0 0 0 0 0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0  
##  0 0 0 0 0 0   0   0   0   0   0   38  43  105 255 253 253 253 253 253 174 6  
##  0 0 0 0 0 0   0   0   0   43  139 224 226 252 253 252 252 252 252 252 252 158
##  0 0 0 0 0 0   0   0   0   178 252 252 252 252 253 252 252 252 252 252 252 252
##  0 0 0 0 0 0   0   0   0   109 252 252 230 132 133 132 132 189 252 252 252 252
##  0 0 0 0 0 0   0   0   0   4   29  29  24  0   0   0   0   14  226 252 252 172
##  0 0 0 0 0 0   0   0   0   0   0   0   0   0   0   0   0   85  243 252 252 144
##  0 0 0 0 0 0   0   0   0   0   0   0   0   0   0   0   88  189 252 252 252 14 
##  0 0 0 0 0 0   0   0   0   0   0   0   0   0   91  212 247 252 252 252 204 9  
##  0 0 0 0 0 0   0   0   0   32  125 193 193 193 253 252 252 252 238 102 28  0  
##  0 0 0 0 0 0   0   0   45  222 252 252 252 252 253 252 252 252 177 0   0   0  
##  0 0 0 0 0 0   0   0   45  223 253 253 253 253 255 253 253 253 253 74  0   0  
##  0 0 0 0 0 0   0   0   0   31  123 52  44  44  44  44  143 252 252 74  0   0  
##  0 0 0 0 0 0   0   0   0   0   0   0   0   0   0   0   15  252 252 74  0   0  
##  0 0 0 0 0 0   0   0   0   0   0   0   0   0   0   0   86  252 252 74  0   0  
##  0 0 0 0 0 0   5   75  9   0   0   0   0   0   0   98  242 252 252 74  0   0  
##  0 0 0 0 0 61  183 252 29  0   0   0   0   18  92  239 252 252 243 65  0   0  
##  0 0 0 0 0 208 252 252 147 134 134 134 134 203 253 252 252 188 83  0   0   0  
##  0 0 0 0 0 208 252 252 252 252 252 252 252 252 253 230 153 8   0   0   0   0  
##  0 0 0 0 0 49  157 252 252 252 252 252 217 207 146 45  0   0   0   0   0   0  
##  0 0 0 0 0 0   7   103 235 252 172 103 24  0   0   0   0   0   0   0   0   0  
##  0 0 0 0 0 0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0  
##  0 0 0 0 0 0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0  
##  0 0 0 0 0 0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0  
##              
##  0  0 0 0 0 0
##  0  0 0 0 0 0
##  0  0 0 0 0 0
##  0  0 0 0 0 0
##  0  0 0 0 0 0
##  0  0 0 0 0 0
##  14 0 0 0 0 0
##  59 0 0 0 0 0
##  59 0 0 0 0 0
##  7  0 0 0 0 0
##  0  0 0 0 0 0
##  0  0 0 0 0 0
##  0  0 0 0 0 0
##  0  0 0 0 0 0
##  0  0 0 0 0 0
##  0  0 0 0 0 0
##  0  0 0 0 0 0
##  0  0 0 0 0 0
##  0  0 0 0 0 0
##  0  0 0 0 0 0
##  0  0 0 0 0 0
##  0  0 0 0 0 0
##  0  0 0 0 0 0
##  0  0 0 0 0 0
##  0  0 0 0 0 0
##  0  0 0 0 0 0
##  0  0 0 0 0 0
##  0  0 0 0 0 0

T-SNE Visual

tsne_v <- Rtsne(as.matrix(select(mnist_subset, -label)), dims=2, check_duplicates = FALSE, pca = TRUE, pca_scale = FALSE, theta = 0.1 , perplexity = 45)
tsne_coords = data.frame(X=tsne_v$Y[,1], Y=tsne_v$Y[,2], Label=as.factor(mnist_subset$label))
ggplot(tsne_coords, aes(x=X, y=Y, color=Label)) + geom_point(size = 1) + scale_color_brewer(palette = "Set3") + theme_dark()

Data Processing

Dimensionality Reduction

Near Zero Variance

x <- nearZeroVar(X, saveMetrics = TRUE)
x

mnist_subset_reduced <- mnist_subset[ , nonzero_var]

PCA

pca <- prcomp(select(mnist_raw, -X1), center = TRUE, scale = FALSE)
summary(pca)

## Importance of components:
##                              PC1       PC2       PC3       PC4       PC5
## Standard deviation     576.82291 493.23822 459.89930 429.85624 408.56680
## Proportion of Variance   0.09705   0.07096   0.06169   0.05389   0.04869
## Cumulative Proportion    0.09705   0.16801   0.22970   0.28359   0.33228
##                              PC6       PC7       PC8       PC9      PC10
## Standard deviation     384.50613 334.93015 314.44305 307.72756 284.27069
## Proportion of Variance   0.04312   0.03272   0.02884   0.02762   0.02357
## Cumulative Proportion    0.37540   0.40812   0.43696   0.46458   0.48815
##                             PC11      PC12      PC13      PC14      PC15
## Standard deviation     268.91192 263.35964 242.54251 240.86113 232.64513
## Proportion of Variance   0.02109   0.02023   0.01716   0.01692   0.01579
## Cumulative Proportion    0.50924   0.52947   0.54663   0.56355   0.57934
##                             PC16      PC17      PC18      PC19      PC20
## Standard deviation     225.48408 213.10234 209.23303 201.75565 198.79585
## Proportion of Variance   0.01483   0.01325   0.01277   0.01187   0.01153
## Cumulative Proportion    0.59417   0.60741   0.62018   0.63205   0.64358
##                             PC21      PC22      PC23      PC24      PC25
## Standard deviation     191.18979 185.78265 180.81285 176.88015 174.03319
## Proportion of Variance   0.01066   0.01007   0.00954   0.00913   0.00883
## Cumulative Proportion    0.65424   0.66431   0.67385   0.68297   0.69180
##                             PC26      PC27      PC28      PC29      PC30
## Standard deviation     169.63513 166.91101 164.19678 159.79109 153.90297
## Proportion of Variance   0.00839   0.00813   0.00786   0.00745   0.00691
## Cumulative Proportion    0.70020   0.70832   0.71619   0.72363   0.73054
##                             PC31      PC32      PC33      PC34     PC35
## Standard deviation     150.20906 149.06967 143.73821 141.81321 139.7970
## Proportion of Variance   0.00658   0.00648   0.00603   0.00587   0.0057
## Cumulative Proportion    0.73712   0.74361   0.74963   0.75550   0.7612
##                             PC36      PC37      PC38      PC39      PC40
## Standard deviation     136.52213 131.68485 129.32998 128.47497 127.24647
## Proportion of Variance   0.00544   0.00506   0.00488   0.00481   0.00472
## Cumulative Proportion    0.76663   0.77169   0.77657   0.78139   0.78611
##                             PC41      PC42      PC43      PC44      PC45
## Standard deviation     125.13824 123.49581 119.78451 116.84530 114.88631
## Proportion of Variance   0.00457   0.00445   0.00419   0.00398   0.00385
## Cumulative Proportion    0.79068   0.79512   0.79931   0.80329   0.80714
##                             PC46      PC47      PC48     PC49      PC50
## Standard deviation     113.40383 111.40683 109.79215 107.9764 105.04987
## Proportion of Variance   0.00375   0.00362   0.00352   0.0034   0.00322
## Cumulative Proportion    0.81089   0.81451   0.81803   0.8214   0.82465
##                             PC51      PC52      PC53     PC54     PC55     PC56
## Standard deviation     104.58254 103.55928 100.73616 99.53299 98.69862 96.46856
## Proportion of Variance   0.00319   0.00313   0.00296  0.00289  0.00284  0.00271
## Cumulative Proportion    0.82784   0.83097   0.83392  0.83681  0.83966  0.84237
##                           PC57     PC58     PC59     PC60     PC61     PC62
## Standard deviation     96.1278 94.13680 93.27664 91.60961 90.80616 90.57116
## Proportion of Variance  0.0027  0.00258  0.00254  0.00245  0.00241  0.00239
## Cumulative Proportion   0.8451  0.84765  0.85019  0.85264  0.85504  0.85743
##                           PC63     PC64     PC65     PC66     PC67     PC68
## Standard deviation     88.8795 87.15064 85.60037 84.28952 83.43454 82.13828
## Proportion of Variance  0.0023  0.00222  0.00214  0.00207  0.00203  0.00197
## Cumulative Proportion   0.8597  0.86195  0.86409  0.86616  0.86819  0.87016
##                            PC69     PC70     PC71     PC72     PC73     PC74
## Standard deviation     81.31394 80.41933 80.06568 78.79362 78.02385 77.43633
## Proportion of Variance  0.00193  0.00189  0.00187  0.00181  0.00178  0.00175
## Cumulative Proportion   0.87209  0.87398  0.87585  0.87766  0.87943  0.88118
##                            PC75     PC76     PC77     PC78     PC79     PC80
## Standard deviation     75.38584 74.96071 74.40250 72.92579 71.14011 70.06279
## Proportion of Variance  0.00166  0.00164  0.00161  0.00155  0.00148  0.00143
## Cumulative Proportion   0.88284  0.88448  0.88609  0.88764  0.88912  0.89055
##                            PC81     PC82    PC83     PC84     PC85     PC86
## Standard deviation     69.79759 69.56609 69.3245 68.21821 67.74181 67.37367
## Proportion of Variance  0.00142  0.00141  0.0014  0.00136  0.00134  0.00132
## Cumulative Proportion   0.89197  0.89338  0.8948  0.89614  0.89748  0.89880
##                           PC87     PC88     PC89     PC90     PC91     PC92
## Standard deviation     66.8015 65.69294 64.89350 64.56413 63.34453 62.75702
## Proportion of Variance  0.0013  0.00126  0.00123  0.00122  0.00117  0.00115
## Cumulative Proportion   0.9001  0.90136  0.90259  0.90381  0.90498  0.90613
##                            PC93     PC94     PC95     PC96     PC97     PC98
## Standard deviation     62.31035 61.65807 61.13207 60.54639 59.76924 59.71496
## Proportion of Variance  0.00113  0.00111  0.00109  0.00107  0.00104  0.00104
## Cumulative Proportion   0.90726  0.90837  0.90946  0.91053  0.91157  0.91261
##                            PC99    PC100    PC101    PC102    PC103    PC104
## Standard deviation     58.92007 58.70758 58.08361 57.06139 56.81012 56.04533
## Proportion of Variance  0.00101  0.00101  0.00098  0.00095  0.00094  0.00092
## Cumulative Proportion   0.91362  0.91463  0.91561  0.91656  0.91750  0.91842
##                           PC105   PC106    PC107    PC108    PC109    PC110
## Standard deviation     55.79040 55.4520 54.47014 54.14757 53.84431 53.10300
## Proportion of Variance  0.00091  0.0009  0.00087  0.00086  0.00085  0.00082
## Cumulative Proportion   0.91933  0.9202  0.92109  0.92195  0.92279  0.92361
##                           PC111    PC112    PC113    PC114    PC115    PC116
## Standard deviation     52.09557 51.90947 51.86546 51.34142 51.18036 50.81304
## Proportion of Variance  0.00079  0.00079  0.00078  0.00077  0.00076  0.00075
## Cumulative Proportion   0.92440  0.92519  0.92598  0.92674  0.92751  0.92826
##                           PC117    PC118    PC119    PC120   PC121    PC122
## Standard deviation     50.25995 49.92969 49.67220 49.22724 48.8289 48.71423
## Proportion of Variance  0.00074  0.00073  0.00072  0.00071  0.0007  0.00069
## Cumulative Proportion   0.92900  0.92973  0.93044  0.93115  0.9318  0.93254
##                           PC123    PC124    PC125    PC126    PC127    PC128
## Standard deviation     48.40152 48.07305 47.81620 47.03504 46.68122 46.53602
## Proportion of Variance  0.00068  0.00067  0.00067  0.00065  0.00064  0.00063
## Cumulative Proportion   0.93322  0.93390  0.93456  0.93521  0.93584  0.93648
##                           PC129    PC130   PC131    PC132    PC133    PC134
## Standard deviation     46.21396 45.55479 45.4907 45.14620 44.91119 44.84290
## Proportion of Variance  0.00062  0.00061  0.0006  0.00059  0.00059  0.00059
## Cumulative Proportion   0.93710  0.93770  0.9383  0.93890  0.93949  0.94008
##                           PC135    PC136    PC137    PC138    PC139    PC140
## Standard deviation     44.64447 44.47098 44.02692 43.61206 42.83495 42.46335
## Proportion of Variance  0.00058  0.00058  0.00057  0.00055  0.00054  0.00053
## Cumulative Proportion   0.94066  0.94124  0.94180  0.94236  0.94289  0.94342
##                           PC141    PC142   PC143   PC144   PC145    PC146
## Standard deviation     42.42975 41.82604 41.5263 41.4481 41.3502 41.03211
## Proportion of Variance  0.00053  0.00051  0.0005  0.0005  0.0005  0.00049
## Cumulative Proportion   0.94394  0.94445  0.9449  0.9455  0.9459  0.94645
##                           PC147    PC148    PC149    PC150    PC151    PC152
## Standard deviation     40.80033 40.68710 40.31324 40.07173 39.99670 39.85625
## Proportion of Variance  0.00049  0.00048  0.00047  0.00047  0.00047  0.00046
## Cumulative Proportion   0.94693  0.94741  0.94789  0.94836  0.94882  0.94929
##                           PC153    PC154    PC155    PC156    PC157    PC158
## Standard deviation     39.68236 39.29540 39.22674 38.95937 38.69288 38.28196
## Proportion of Variance  0.00046  0.00045  0.00045  0.00044  0.00044  0.00043
## Cumulative Proportion   0.94975  0.95020  0.95064  0.95109  0.95152  0.95195
##                           PC159    PC160    PC161    PC162   PC163   PC164
## Standard deviation     38.17122 38.03775 37.74373 37.47588 37.0705 36.9343
## Proportion of Variance  0.00042  0.00042  0.00042  0.00041  0.0004  0.0004
## Cumulative Proportion   0.95238  0.95280  0.95321  0.95362  0.9540  0.9544
##                           PC165    PC166    PC167    PC168    PC169    PC170
## Standard deviation     36.75740 36.58973 36.37165 36.08414 36.04243 35.80575
## Proportion of Variance  0.00039  0.00039  0.00039  0.00038  0.00038  0.00037
## Cumulative Proportion   0.95482  0.95521  0.95559  0.95597  0.95635  0.95673
##                           PC171    PC172    PC173    PC174    PC175    PC176
## Standard deviation     35.58846 35.38248 35.31763 35.12593 34.86964 34.78378
## Proportion of Variance  0.00037  0.00037  0.00036  0.00036  0.00035  0.00035
## Cumulative Proportion   0.95709  0.95746  0.95782  0.95818  0.95854  0.95889
##                           PC177    PC178    PC179    PC180    PC181    PC182
## Standard deviation     34.50622 34.44002 34.21479 34.02380 33.99599 33.64795
## Proportion of Variance  0.00035  0.00035  0.00034  0.00034  0.00034  0.00033
## Cumulative Proportion   0.95924  0.95958  0.95993  0.96026  0.96060  0.96093
##                           PC183    PC184    PC185    PC186    PC187    PC188
## Standard deviation     33.55243 33.42715 33.27486 33.18843 32.98274 32.92426
## Proportion of Variance  0.00033  0.00033  0.00032  0.00032  0.00032  0.00032
## Cumulative Proportion   0.96126  0.96159  0.96191  0.96223  0.96255  0.96286
##                           PC189    PC190    PC191   PC192   PC193   PC194
## Standard deviation     32.65069 32.57680 32.38837 32.2456 32.1996 32.1548
## Proportion of Variance  0.00031  0.00031  0.00031  0.0003  0.0003  0.0003
## Cumulative Proportion   0.96317  0.96348  0.96379  0.9641  0.9644  0.9647
##                          PC195   PC196    PC197    PC198    PC199    PC200
## Standard deviation     31.8945 31.8429 31.72016 31.67694 31.40954 31.22304
## Proportion of Variance  0.0003  0.0003  0.00029  0.00029  0.00029  0.00028
## Cumulative Proportion   0.9650  0.9653  0.96558  0.96588  0.96616  0.96645
##                           PC201    PC202    PC203    PC204    PC205    PC206
## Standard deviation     31.09215 30.93207 30.74665 30.56436 30.39341 30.29580
## Proportion of Variance  0.00028  0.00028  0.00028  0.00027  0.00027  0.00027
## Cumulative Proportion   0.96673  0.96701  0.96728  0.96756  0.96783  0.96809
##                           PC207    PC208    PC209    PC210    PC211    PC212
## Standard deviation     30.14904 30.10392 29.96442 29.82496 29.77353 29.67929
## Proportion of Variance  0.00027  0.00026  0.00026  0.00026  0.00026  0.00026
## Cumulative Proportion   0.96836  0.96862  0.96889  0.96914  0.96940  0.96966
##                           PC213    PC214    PC215    PC216    PC217    PC218
## Standard deviation     29.55277 29.44090 29.41048 29.25110 29.18972 29.04597
## Proportion of Variance  0.00025  0.00025  0.00025  0.00025  0.00025  0.00025
## Cumulative Proportion   0.96991  0.97017  0.97042  0.97067  0.97092  0.97116
##                           PC219    PC220    PC221    PC222    PC223    PC224
## Standard deviation     28.92058 28.79632 28.74300 28.71172 28.58161 28.49653
## Proportion of Variance  0.00024  0.00024  0.00024  0.00024  0.00024  0.00024
## Cumulative Proportion   0.97141  0.97165  0.97189  0.97213  0.97237  0.97261
##                           PC225    PC226    PC227    PC228    PC229    PC230
## Standard deviation     28.23818 28.16102 28.08905 27.96419 27.93331 27.63673
## Proportion of Variance  0.00023  0.00023  0.00023  0.00023  0.00023  0.00022
## Cumulative Proportion   0.97284  0.97307  0.97330  0.97353  0.97376  0.97398
##                           PC231    PC232    PC233    PC234    PC235    PC236
## Standard deviation     27.58003 27.49158 27.33050 27.25449 27.16045 27.09922
## Proportion of Variance  0.00022  0.00022  0.00022  0.00022  0.00022  0.00021
## Cumulative Proportion   0.97420  0.97442  0.97464  0.97486  0.97507  0.97528
##                           PC237    PC238    PC239    PC240   PC241   PC242
## Standard deviation     27.01434 26.85633 26.81689 26.63772 26.4933 26.4640
## Proportion of Variance  0.00021  0.00021  0.00021  0.00021  0.0002  0.0002
## Cumulative Proportion   0.97550  0.97571  0.97592  0.97612  0.9763  0.9765
##                          PC243   PC244   PC245   PC246   PC247    PC248
## Standard deviation     26.2718 26.2503 26.1201 26.0303 25.9113 25.77203
## Proportion of Variance  0.0002  0.0002  0.0002  0.0002  0.0002  0.00019
## Cumulative Proportion   0.9767  0.9769  0.9771  0.9773  0.9775  0.97772
##                           PC249    PC250    PC251    PC252    PC253    PC254
## Standard deviation     25.71411 25.64163 25.62620 25.46082 25.38900 25.27937
## Proportion of Variance  0.00019  0.00019  0.00019  0.00019  0.00019  0.00019
## Cumulative Proportion   0.97792  0.97811  0.97830  0.97849  0.97868  0.97886
##                           PC255    PC256    PC257    PC258    PC259    PC260
## Standard deviation     25.24240 25.15877 25.09585 25.05828 24.97155 24.83289
## Proportion of Variance  0.00019  0.00018  0.00018  0.00018  0.00018  0.00018
## Cumulative Proportion   0.97905  0.97923  0.97942  0.97960  0.97978  0.97996
##                           PC261    PC262    PC263    PC264    PC265    PC266
## Standard deviation     24.67287 24.61255 24.56189 24.41634 24.38443 24.32931
## Proportion of Variance  0.00018  0.00018  0.00018  0.00017  0.00017  0.00017
## Cumulative Proportion   0.98014  0.98032  0.98049  0.98067  0.98084  0.98101
##                           PC267    PC268    PC269    PC270    PC271    PC272
## Standard deviation     24.24038 24.14478 24.09247 24.00015 23.92262 23.88895
## Proportion of Variance  0.00017  0.00017  0.00017  0.00017  0.00017  0.00017
## Cumulative Proportion   0.98118  0.98135  0.98152  0.98169  0.98186  0.98202
##                           PC273    PC274    PC275    PC276    PC277    PC278
## Standard deviation     23.72719 23.67482 23.61511 23.55933 23.48651 23.40993
## Proportion of Variance  0.00016  0.00016  0.00016  0.00016  0.00016  0.00016
## Cumulative Proportion   0.98219  0.98235  0.98251  0.98268  0.98284  0.98300
##                           PC279    PC280    PC281    PC282    PC283    PC284
## Standard deviation     23.33579 23.22673 23.18137 23.04485 22.96377 22.90838
## Proportion of Variance  0.00016  0.00016  0.00016  0.00015  0.00015  0.00015
## Cumulative Proportion   0.98316  0.98331  0.98347  0.98362  0.98378  0.98393
##                           PC285    PC286    PC287    PC288    PC289    PC290
## Standard deviation     22.83124 22.67845 22.65230 22.54339 22.50112 22.38299
## Proportion of Variance  0.00015  0.00015  0.00015  0.00015  0.00015  0.00015
## Cumulative Proportion   0.98408  0.98423  0.98438  0.98453  0.98468  0.98482
##                           PC291    PC292    PC293    PC294    PC295    PC296
## Standard deviation     22.26973 22.16027 22.12072 22.07002 22.04511 21.94716
## Proportion of Variance  0.00014  0.00014  0.00014  0.00014  0.00014  0.00014
## Cumulative Proportion   0.98497  0.98511  0.98526  0.98540  0.98554  0.98568
##                           PC297    PC298    PC299    PC300    PC301    PC302
## Standard deviation     21.77035 21.74944 21.66012 21.59148 21.57717 21.47475
## Proportion of Variance  0.00014  0.00014  0.00014  0.00014  0.00014  0.00013
## Cumulative Proportion   0.98582  0.98596  0.98609  0.98623  0.98636  0.98650
##                           PC303    PC304    PC305    PC306    PC307    PC308
## Standard deviation     21.40343 21.30112 21.16864 21.12712 21.04364 20.95784
## Proportion of Variance  0.00013  0.00013  0.00013  0.00013  0.00013  0.00013
## Cumulative Proportion   0.98663  0.98677  0.98690  0.98703  0.98716  0.98728
##                           PC309    PC310    PC311    PC312    PC313    PC314
## Standard deviation     20.90903 20.88461 20.81232 20.77286 20.67945 20.58236
## Proportion of Variance  0.00013  0.00013  0.00013  0.00013  0.00012  0.00012
## Cumulative Proportion   0.98741  0.98754  0.98766  0.98779  0.98791  0.98804
##                           PC315    PC316    PC317    PC318    PC319    PC320
## Standard deviation     20.50388 20.41552 20.37012 20.28075 20.23460 20.19446
## Proportion of Variance  0.00012  0.00012  0.00012  0.00012  0.00012  0.00012
## Cumulative Proportion   0.98816  0.98828  0.98840  0.98852  0.98864  0.98876
##                           PC321    PC322    PC323    PC324    PC325    PC326
## Standard deviation     20.10286 20.03348 19.98507 19.95109 19.81586 19.78871
## Proportion of Variance  0.00012  0.00012  0.00012  0.00012  0.00011  0.00011
## Cumulative Proportion   0.98888  0.98900  0.98911  0.98923  0.98934  0.98946
##                           PC327    PC328    PC329    PC330    PC331    PC332
## Standard deviation     19.63125 19.53222 19.46759 19.45796 19.42516 19.32927
## Proportion of Variance  0.00011  0.00011  0.00011  0.00011  0.00011  0.00011
## Cumulative Proportion   0.98957  0.98968  0.98979  0.98990  0.99001  0.99012
##                           PC333    PC334    PC335    PC336    PC337    PC338
## Standard deviation     19.27138 19.20656 19.15371 19.08203 19.05723 18.97798
## Proportion of Variance  0.00011  0.00011  0.00011  0.00011  0.00011  0.00011
## Cumulative Proportion   0.99023  0.99034  0.99044  0.99055  0.99066  0.99076
##                          PC339   PC340   PC341   PC342   PC343   PC344   PC345
## Standard deviation     18.8302 18.7234 18.7047 18.5944 18.5407 18.4789 18.4581
## Proportion of Variance  0.0001  0.0001  0.0001  0.0001  0.0001  0.0001  0.0001
## Cumulative Proportion   0.9909  0.9910  0.9911  0.9912  0.9913  0.9914  0.9915
##                          PC346   PC347   PC348   PC349    PC350    PC351
## Standard deviation     18.3418 18.2922 18.2205 18.0971 18.00409 17.93286
## Proportion of Variance  0.0001  0.0001  0.0001  0.0001  0.00009  0.00009
## Cumulative Proportion   0.9916  0.9917  0.9918  0.9919  0.99195  0.99205
##                           PC352    PC353    PC354    PC355    PC356    PC357
## Standard deviation     17.92564 17.80766 17.72027 17.70181 17.63170 17.57336
## Proportion of Variance  0.00009  0.00009  0.00009  0.00009  0.00009  0.00009
## Cumulative Proportion   0.99214  0.99223  0.99232  0.99242  0.99251  0.99260
##                           PC358    PC359    PC360    PC361    PC362    PC363
## Standard deviation     17.50448 17.48346 17.39553 17.30329 17.20211 17.12879
## Proportion of Variance  0.00009  0.00009  0.00009  0.00009  0.00009  0.00009
## Cumulative Proportion   0.99269  0.99277  0.99286  0.99295  0.99304  0.99312
##                           PC364    PC365    PC366    PC367    PC368    PC369
## Standard deviation     17.08694 17.03110 16.90083 16.86684 16.81806 16.75810
## Proportion of Variance  0.00009  0.00008  0.00008  0.00008  0.00008  0.00008
## Cumulative Proportion   0.99321  0.99329  0.99338  0.99346  0.99354  0.99362
##                           PC370    PC371    PC372    PC373    PC374    PC375
## Standard deviation     16.67389 16.65463 16.52085 16.50628 16.41858 16.35949
## Proportion of Variance  0.00008  0.00008  0.00008  0.00008  0.00008  0.00008
## Cumulative Proportion   0.99370  0.99378  0.99386  0.99394  0.99402  0.99410
##                           PC376    PC377    PC378    PC379    PC380    PC381
## Standard deviation     16.29272 16.20462 16.12123 15.99373 15.91164 15.88181
## Proportion of Variance  0.00008  0.00008  0.00008  0.00007  0.00007  0.00007
## Cumulative Proportion   0.99418  0.99425  0.99433  0.99440  0.99448  0.99455
##                           PC382    PC383    PC384    PC385    PC386    PC387
## Standard deviation     15.80027 15.78924 15.71881 15.64773 15.58069 15.45318
## Proportion of Variance  0.00007  0.00007  0.00007  0.00007  0.00007  0.00007
## Cumulative Proportion   0.99463  0.99470  0.99477  0.99484  0.99491  0.99498
##                           PC388    PC389    PC390    PC391    PC392    PC393
## Standard deviation     15.40801 15.30246 15.26194 15.16280 15.06934 14.95609
## Proportion of Variance  0.00007  0.00007  0.00007  0.00007  0.00007  0.00007
## Cumulative Proportion   0.99505  0.99512  0.99519  0.99525  0.99532  0.99539
##                           PC394    PC395    PC396    PC397    PC398    PC399
## Standard deviation     14.83639 14.77942 14.74960 14.58353 14.49556 14.38068
## Proportion of Variance  0.00006  0.00006  0.00006  0.00006  0.00006  0.00006
## Cumulative Proportion   0.99545  0.99551  0.99558  0.99564  0.99570  0.99576
##                           PC400    PC401    PC402    PC403    PC404    PC405
## Standard deviation     14.33403 14.26687 14.18998 14.17678 14.02672 13.94572
## Proportion of Variance  0.00006  0.00006  0.00006  0.00006  0.00006  0.00006
## Cumulative Proportion   0.99582  0.99588  0.99594  0.99600  0.99605  0.99611
##                           PC406    PC407    PC408    PC409    PC410    PC411
## Standard deviation     13.90244 13.80874 13.71967 13.69935 13.66605 13.52193
## Proportion of Variance  0.00006  0.00006  0.00005  0.00005  0.00005  0.00005
## Cumulative Proportion   0.99617  0.99622  0.99628  0.99633  0.99639  0.99644
##                           PC412    PC413    PC414    PC415    PC416    PC417
## Standard deviation     13.44889 13.42237 13.37766 13.33323 13.22515 13.14274
## Proportion of Variance  0.00005  0.00005  0.00005  0.00005  0.00005  0.00005
## Cumulative Proportion   0.99649  0.99655  0.99660  0.99665  0.99670  0.99675
##                           PC418    PC419    PC420    PC421    PC422    PC423
## Standard deviation     13.06088 12.97026 12.92172 12.82068 12.72217 12.64817
## Proportion of Variance  0.00005  0.00005  0.00005  0.00005  0.00005  0.00005
## Cumulative Proportion   0.99680  0.99685  0.99690  0.99695  0.99699  0.99704
##                           PC424    PC425    PC426    PC427    PC428    PC429
## Standard deviation     12.61858 12.50381 12.40855 12.24221 12.13152 12.08657
## Proportion of Variance  0.00005  0.00005  0.00004  0.00004  0.00004  0.00004
## Cumulative Proportion   0.99709  0.99713  0.99718  0.99722  0.99726  0.99731
##                           PC430    PC431    PC432    PC433    PC434    PC435
## Standard deviation     12.05484 12.02474 11.96909 11.91199 11.83390 11.75260
## Proportion of Variance  0.00004  0.00004  0.00004  0.00004  0.00004  0.00004
## Cumulative Proportion   0.99735  0.99739  0.99743  0.99747  0.99752  0.99756
##                           PC436    PC437    PC438    PC439    PC440    PC441
## Standard deviation     11.50224 11.45195 11.43545 11.39670 11.31215 11.27427
## Proportion of Variance  0.00004  0.00004  0.00004  0.00004  0.00004  0.00004
## Cumulative Proportion   0.99759  0.99763  0.99767  0.99771  0.99775  0.99778
##                           PC442    PC443    PC444    PC445    PC446    PC447
## Standard deviation     11.20042 11.07339 11.01021 10.99388 10.93398 10.88378
## Proportion of Variance  0.00004  0.00004  0.00004  0.00004  0.00003  0.00003
## Cumulative Proportion   0.99782  0.99786  0.99789  0.99793  0.99796  0.99800
##                           PC448    PC449    PC450    PC451    PC452    PC453
## Standard deviation     10.85745 10.70144 10.63662 10.50679 10.46442 10.41540
## Proportion of Variance  0.00003  0.00003  0.00003  0.00003  0.00003  0.00003
## Cumulative Proportion   0.99803  0.99806  0.99810  0.99813  0.99816  0.99819
##                           PC454    PC455    PC456    PC457    PC458   PC459
## Standard deviation     10.31156 10.21069 10.16809 10.12589 10.00781 9.94830
## Proportion of Variance  0.00003  0.00003  0.00003  0.00003  0.00003 0.00003
## Cumulative Proportion   0.99822  0.99825  0.99828  0.99831  0.99834 0.99837
##                          PC460   PC461   PC462   PC463   PC464   PC465   PC466
## Standard deviation     9.88815 9.80924 9.78165 9.71304 9.61156 9.40008 9.31203
## Proportion of Variance 0.00003 0.00003 0.00003 0.00003 0.00003 0.00003 0.00003
## Cumulative Proportion  0.99840 0.99843 0.99846 0.99848 0.99851 0.99854 0.99856
##                          PC467   PC468   PC469   PC470   PC471   PC472   PC473
## Standard deviation     9.28824 9.22625 9.20007 9.13817 9.13023 9.11799 9.01909
## Proportion of Variance 0.00003 0.00002 0.00002 0.00002 0.00002 0.00002 0.00002
## Cumulative Proportion  0.99859 0.99861 0.99864 0.99866 0.99869 0.99871 0.99873
##                          PC474   PC475   PC476   PC477   PC478   PC479   PC480
## Standard deviation     8.94763 8.92187 8.78660 8.75667 8.70822 8.57211 8.54468
## Proportion of Variance 0.00002 0.00002 0.00002 0.00002 0.00002 0.00002 0.00002
## Cumulative Proportion  0.99876 0.99878 0.99880 0.99882 0.99885 0.99887 0.99889
##                          PC481   PC482   PC483   PC484   PC485   PC486   PC487
## Standard deviation     8.49880 8.45611 8.35841 8.29926 8.25260 8.20261 8.17962
## Proportion of Variance 0.00002 0.00002 0.00002 0.00002 0.00002 0.00002 0.00002
## Cumulative Proportion  0.99891 0.99893 0.99895 0.99897 0.99899 0.99901 0.99903
##                          PC488   PC489   PC490   PC491   PC492   PC493   PC494
## Standard deviation     8.06245 7.95324 7.89638 7.87810 7.84645 7.76543 7.72594
## Proportion of Variance 0.00002 0.00002 0.00002 0.00002 0.00002 0.00002 0.00002
## Cumulative Proportion  0.99905 0.99907 0.99909 0.99910 0.99912 0.99914 0.99916
##                          PC495   PC496   PC497   PC498   PC499   PC500   PC501
## Standard deviation     7.48897 7.44644 7.43153 7.38422 7.29903 7.22036 7.21112
## Proportion of Variance 0.00002 0.00002 0.00002 0.00002 0.00002 0.00002 0.00002
## Cumulative Proportion  0.99917 0.99919 0.99921 0.99922 0.99924 0.99925 0.99927
##                          PC502   PC503   PC504   PC505   PC506   PC507   PC508
## Standard deviation     7.16640 7.11333 7.06977 7.01534 6.97181 6.91876 6.87591
## Proportion of Variance 0.00001 0.00001 0.00001 0.00001 0.00001 0.00001 0.00001
## Cumulative Proportion  0.99928 0.99930 0.99931 0.99933 0.99934 0.99935 0.99937
##                          PC509   PC510   PC511   PC512   PC513   PC514   PC515
## Standard deviation     6.84714 6.76112 6.70275 6.55922 6.51533 6.46353 6.41462
## Proportion of Variance 0.00001 0.00001 0.00001 0.00001 0.00001 0.00001 0.00001
## Cumulative Proportion  0.99938 0.99940 0.99941 0.99942 0.99943 0.99945 0.99946
##                          PC516   PC517   PC518   PC519   PC520   PC521   PC522
## Standard deviation     6.39079 6.35422 6.34184 6.24598 6.22895 6.20952 6.14186
## Proportion of Variance 0.00001 0.00001 0.00001 0.00001 0.00001 0.00001 0.00001
## Cumulative Proportion  0.99947 0.99948 0.99949 0.99950 0.99952 0.99953 0.99954
##                          PC523   PC524   PC525   PC526   PC527   PC528   PC529
## Standard deviation     6.12353 6.01993 6.01090 5.92288 5.86175 5.72952 5.66865
## Proportion of Variance 0.00001 0.00001 0.00001 0.00001 0.00001 0.00001 0.00001
## Cumulative Proportion  0.99955 0.99956 0.99957 0.99958 0.99959 0.99960 0.99961
##                          PC530   PC531   PC532   PC533   PC534   PC535   PC536
## Standard deviation     5.63464 5.62304 5.51511 5.49585 5.42759 5.28935 5.22732
## Proportion of Variance 0.00001 0.00001 0.00001 0.00001 0.00001 0.00001 0.00001
## Cumulative Proportion  0.99962 0.99963 0.99964 0.99965 0.99965 0.99966 0.99967
##                          PC537   PC538   PC539   PC540   PC541   PC542   PC543
## Standard deviation     5.19352 5.14935 5.11929 5.09702 5.08166 5.02626 4.99224
## Proportion of Variance 0.00001 0.00001 0.00001 0.00001 0.00001 0.00001 0.00001
## Cumulative Proportion  0.99968 0.99969 0.99969 0.99970 0.99971 0.99972 0.99972
##                          PC544   PC545   PC546   PC547   PC548   PC549   PC550
## Standard deviation     4.95068 4.88548 4.85715 4.79830 4.77697 4.73451 4.70986
## Proportion of Variance 0.00001 0.00001 0.00001 0.00001 0.00001 0.00001 0.00001
## Cumulative Proportion  0.99973 0.99974 0.99974 0.99975 0.99976 0.99976 0.99977
##                          PC551   PC552   PC553   PC554   PC555   PC556   PC557
## Standard deviation     4.58810 4.50800 4.43123 4.41185 4.36409 4.28256 4.25097
## Proportion of Variance 0.00001 0.00001 0.00001 0.00001 0.00001 0.00001 0.00001
## Cumulative Proportion  0.99978 0.99978 0.99979 0.99979 0.99980 0.99981 0.99981
##                          PC558   PC559  PC560  PC561  PC562  PC563  PC564
## Standard deviation     4.21526 4.15656 4.1385 4.1124 4.0991 4.0612 4.0125
## Proportion of Variance 0.00001 0.00001 0.0000 0.0000 0.0000 0.0000 0.0000
## Cumulative Proportion  0.99982 0.99982 0.9998 0.9998 0.9998 0.9998 0.9998
##                         PC565  PC566  PC567  PC568  PC569  PC570  PC571  PC572
## Standard deviation     3.9813 3.9516 3.7939 3.7879 3.7091 3.7040 3.6778 3.6094
## Proportion of Variance 0.0000 0.0000 0.0000 0.0000 0.0000 0.0000 0.0000 0.0000
## Cumulative Proportion  0.9999 0.9999 0.9999 0.9999 0.9999 0.9999 0.9999 0.9999
##                         PC573  PC574  PC575  PC576  PC577  PC578  PC579  PC580
## Standard deviation     3.6004 3.3698 3.3318 3.3188 3.2994 3.2704 3.2507 3.2209
## Proportion of Variance 0.0000 0.0000 0.0000 0.0000 0.0000 0.0000 0.0000 0.0000
## Cumulative Proportion  0.9999 0.9999 0.9999 0.9999 0.9999 0.9999 0.9999 0.9999
##                         PC581  PC582  PC583  PC584  PC585  PC586  PC587  PC588
## Standard deviation     3.2136 3.1914 3.1835 3.1665 3.1004 3.0019 2.9568 2.8455
## Proportion of Variance 0.0000 0.0000 0.0000 0.0000 0.0000 0.0000 0.0000 0.0000
## Cumulative Proportion  0.9999 0.9999 0.9999 0.9999 0.9999 0.9999 0.9999 0.9999
##                         PC589  PC590  PC591  PC592  PC593  PC594  PC595  PC596
## Standard deviation     2.8176 2.7854 2.7742 2.7512 2.7157 2.7129 2.6846 2.6458
## Proportion of Variance 0.0000 0.0000 0.0000 0.0000 0.0000 0.0000 0.0000 0.0000
## Cumulative Proportion  0.9999 0.9999 0.9999 0.9999 0.9999 0.9999 0.9999 0.9999
##                        PC597 PC598 PC599 PC600 PC601 PC602 PC603 PC604 PC605
## Standard deviation     2.538 2.529  2.52 2.515 2.484 2.452 2.356 2.346  2.27
## Proportion of Variance 0.000 0.000  0.00 0.000 0.000 0.000 0.000 0.000  0.00
## Cumulative Proportion  1.000 1.000  1.00 1.000 1.000 1.000 1.000 1.000  1.00
##                        PC606 PC607 PC608 PC609 PC610 PC611 PC612 PC613 PC614
## Standard deviation     2.238 2.201 2.182 2.126 2.114 2.099 2.068 1.959 1.923
## Proportion of Variance 0.000 0.000 0.000 0.000 0.000 0.000 0.000 0.000 0.000
## Cumulative Proportion  1.000 1.000 1.000 1.000 1.000 1.000 1.000 1.000 1.000
##                        PC615 PC616 PC617 PC618 PC619 PC620 PC621 PC622 PC623
## Standard deviation     1.895  1.89 1.884 1.834  1.82  1.79 1.772 1.766 1.756
## Proportion of Variance 0.000  0.00 0.000 0.000  0.00  0.00 0.000 0.000 0.000
## Cumulative Proportion  1.000  1.00 1.000 1.000  1.00  1.00 1.000 1.000 1.000
##                        PC624 PC625 PC626 PC627 PC628 PC629 PC630 PC631 PC632
## Standard deviation     1.732  1.69 1.631 1.598 1.573 1.512 1.504 1.478 1.464
## Proportion of Variance 0.000  0.00 0.000 0.000 0.000 0.000 0.000 0.000 0.000
## Cumulative Proportion  1.000  1.00 1.000 1.000 1.000 1.000 1.000 1.000 1.000
##                        PC633 PC634 PC635 PC636 PC637 PC638 PC639 PC640 PC641
## Standard deviation     1.464 1.423 1.414 1.394 1.387 1.346 1.338 1.279 1.272
## Proportion of Variance 0.000 0.000 0.000 0.000 0.000 0.000 0.000 0.000 0.000
## Cumulative Proportion  1.000 1.000 1.000 1.000 1.000 1.000 1.000 1.000 1.000
##                        PC642 PC643 PC644 PC645 PC646 PC647 PC648 PC649 PC650
## Standard deviation     1.264 1.217 1.201 1.181 1.147 1.114 1.106 1.035 1.009
## Proportion of Variance 0.000 0.000 0.000 0.000 0.000 0.000 0.000 0.000 0.000
## Cumulative Proportion  1.000 1.000 1.000 1.000 1.000 1.000 1.000 1.000 1.000
##                        PC651  PC652  PC653 PC654  PC655 PC656  PC657 PC658
## Standard deviation     0.977 0.9669 0.9396 0.936 0.9095 0.899 0.8879 0.879
## Proportion of Variance 0.000 0.0000 0.0000 0.000 0.0000 0.000 0.0000 0.000
## Cumulative Proportion  1.000 1.0000 1.0000 1.000 1.0000 1.000 1.0000 1.000
##                         PC659  PC660 PC661  PC662  PC663  PC664  PC665  PC666
## Standard deviation     0.8586 0.8333 0.831 0.8201 0.7784 0.7662 0.7195 0.7165
## Proportion of Variance 0.0000 0.0000 0.000 0.0000 0.0000 0.0000 0.0000 0.0000
## Cumulative Proportion  1.0000 1.0000 1.000 1.0000 1.0000 1.0000 1.0000 1.0000
##                         PC667  PC668  PC669  PC670  PC671  PC672  PC673  PC674
## Standard deviation     0.6844 0.6811 0.6626 0.6058 0.5692 0.5488 0.5343 0.5292
## Proportion of Variance 0.0000 0.0000 0.0000 0.0000 0.0000 0.0000 0.0000 0.0000
## Cumulative Proportion  1.0000 1.0000 1.0000 1.0000 1.0000 1.0000 1.0000 1.0000
##                         PC675  PC676  PC677 PC678  PC679  PC680  PC681  PC682
## Standard deviation     0.4846 0.4801 0.4692 0.452 0.4358 0.3998 0.3968 0.3733
## Proportion of Variance 0.0000 0.0000 0.0000 0.000 0.0000 0.0000 0.0000 0.0000
## Cumulative Proportion  1.0000 1.0000 1.0000 1.000 1.0000 1.0000 1.0000 1.0000
##                         PC683  PC684  PC685  PC686  PC687 PC688  PC689 PC690
## Standard deviation     0.3673 0.3515 0.3498 0.3333 0.3254 0.321 0.3148 0.287
## Proportion of Variance 0.0000 0.0000 0.0000 0.0000 0.0000 0.000 0.0000 0.000
## Cumulative Proportion  1.0000 1.0000 1.0000 1.0000 1.0000 1.000 1.0000 1.000
##                         PC691  PC692  PC693  PC694  PC695 PC696  PC697  PC698
## Standard deviation     0.2542 0.2465 0.2388 0.2357 0.1955 0.188 0.1861 0.1722
## Proportion of Variance 0.0000 0.0000 0.0000 0.0000 0.0000 0.000 0.0000 0.0000
## Cumulative Proportion  1.0000 1.0000 1.0000 1.0000 1.0000 1.000 1.0000 1.0000
##                         PC699 PC700  PC701  PC702  PC703  PC704  PC705   PC706
## Standard deviation     0.1652 0.154 0.1536 0.1365 0.1305 0.1126 0.1018 0.08156
## Proportion of Variance 0.0000 0.000 0.0000 0.0000 0.0000 0.0000 0.0000 0.00000
## Cumulative Proportion  1.0000 1.000 1.0000 1.0000 1.0000 1.0000 1.0000 1.00000
##                          PC707   PC708   PC709   PC710   PC711    PC712
## Standard deviation     0.05453 0.04928 0.04872 0.02948 0.02846 0.007244
## Proportion of Variance 0.00000 0.00000 0.00000 0.00000 0.00000 0.000000
## Cumulative Proportion  1.00000 1.00000 1.00000 1.00000 1.00000 1.000000
##                            PC713    PC714     PC715     PC716     PC717
## Standard deviation     1.123e-12 9.66e-13 4.953e-13 4.379e-13 3.889e-13
## Proportion of Variance 0.000e+00 0.00e+00 0.000e+00 0.000e+00 0.000e+00
## Cumulative Proportion  1.000e+00 1.00e+00 1.000e+00 1.000e+00 1.000e+00
##                            PC718     PC719     PC720    PC721     PC722
## Standard deviation     3.643e-13 2.957e-13 2.743e-13 2.44e-13 2.154e-13
## Proportion of Variance 0.000e+00 0.000e+00 0.000e+00 0.00e+00 0.000e+00
## Cumulative Proportion  1.000e+00 1.000e+00 1.000e+00 1.00e+00 1.000e+00
##                            PC723     PC724     PC725     PC726    PC727
## Standard deviation     2.007e-13 1.797e-13 1.764e-13 1.595e-13 1.53e-13
## Proportion of Variance 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.00e+00
## Cumulative Proportion  1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.00e+00
##                            PC728     PC729     PC730     PC731     PC732
## Standard deviation     1.391e-13 1.331e-13 1.257e-13 1.242e-13 1.106e-13
## Proportion of Variance 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00
## Cumulative Proportion  1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00
##                            PC733     PC734     PC735    PC736     PC737
## Standard deviation     1.055e-13 9.396e-14 8.811e-14 8.12e-14 7.436e-14
## Proportion of Variance 0.000e+00 0.000e+00 0.000e+00 0.00e+00 0.000e+00
## Cumulative Proportion  1.000e+00 1.000e+00 1.000e+00 1.00e+00 1.000e+00
##                            PC738     PC739     PC740     PC741     PC742
## Standard deviation     6.845e-14 5.691e-14 5.517e-14 4.973e-14 4.118e-14
## Proportion of Variance 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00
## Cumulative Proportion  1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00
##                            PC743     PC744     PC745     PC746     PC747
## Standard deviation     3.687e-14 3.463e-14 3.463e-14 3.463e-14 3.463e-14
## Proportion of Variance 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00
## Cumulative Proportion  1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00
##                            PC748     PC749     PC750     PC751     PC752
## Standard deviation     3.463e-14 3.463e-14 3.463e-14 3.463e-14 3.463e-14
## Proportion of Variance 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00
## Cumulative Proportion  1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00
##                            PC753     PC754     PC755     PC756     PC757
## Standard deviation     3.463e-14 3.463e-14 3.463e-14 3.463e-14 3.463e-14
## Proportion of Variance 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00
## Cumulative Proportion  1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00
##                            PC758     PC759     PC760     PC761     PC762
## Standard deviation     3.463e-14 3.463e-14 3.463e-14 3.463e-14 3.463e-14
## Proportion of Variance 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00
## Cumulative Proportion  1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00
##                            PC763     PC764     PC765     PC766     PC767
## Standard deviation     3.463e-14 3.463e-14 3.463e-14 3.463e-14 3.463e-14
## Proportion of Variance 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00
## Cumulative Proportion  1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00
##                            PC768     PC769     PC770     PC771     PC772
## Standard deviation     3.463e-14 3.463e-14 3.463e-14 3.463e-14 3.463e-14
## Proportion of Variance 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00
## Cumulative Proportion  1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00
##                            PC773     PC774     PC775     PC776     PC777
## Standard deviation     3.463e-14 3.463e-14 3.463e-14 3.463e-14 3.463e-14
## Proportion of Variance 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00
## Cumulative Proportion  1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00
##                            PC778     PC779     PC780     PC781     PC782
## Standard deviation     3.463e-14 3.432e-14 3.014e-14 2.408e-14 2.107e-14
## Proportion of Variance 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00
## Cumulative Proportion  1.000e+00 1.000e+00 1.000e+00 1.000e+00 1.000e+00
##                            PC783     PC784
## Standard deviation     1.269e-14 3.167e-15
## Proportion of Variance 0.000e+00 0.000e+00
## Cumulative Proportion  1.000e+00 1.000e+00

qqnorm(pca[["x"]][,1])

pca %>% get_eigenvalue() %>% head(10)

pca %>% fviz_eig(addlabels = TRUE)

get_eigenvalue(pca) %>% filter(eigenvalue > 1)

reduced_dim_95 <- get_eigenvalue(pca) %>% filter(cumulative.variance.percent < 95.02)
reduced_dim_95

#mnist_pca_reduced <- cbind(as.data.frame(pca$x[,c(1:nrow(reduced_dim_95))]), Label = mnist_raw$X1)
mnist_pca_reduced <- as.data.frame(pca$x[,c(1:nrow(reduced_dim_95))])
head(mnist_pca_reduced)

Modeling

KNN

# Data Partitioning
set.seed(622)
trainIndex <- createDataPartition(mnist_subset_reduced$label, p = .8, list = FALSE, times = 1)
X_train <- X[trainIndex,]

## Warning: The `i` argument of ``[`()` can't be a matrix as of tibble 3.0.0.
## Convert to a vector.
## This warning is displayed once every 8 hours.
## Call `lifecycle::last_warnings()` to see where this warning was generated.

y_train <- y[trainIndex]
X_test <- X[-trainIndex,]
y_test <- y[-trainIndex]
y_train <- as.factor(y_train)
y_test <- as.factor(y_test)

trControl <- trainControl(method  = "cv", number  = 3)
knn.fit <- train(X_train, y_train,
             method     = "knn",
             tuneGrid   = expand.grid(k = 1:5),
             trControl  = trControl)

## Warning: Setting row names on a tibble is deprecated.

## Warning: Setting row names on a tibble is deprecated.

## Warning: Setting row names on a tibble is deprecated.

## Warning: Setting row names on a tibble is deprecated.

## Warning: Setting row names on a tibble is deprecated.

## Warning: Setting row names on a tibble is deprecated.

## Warning: Setting row names on a tibble is deprecated.

## Warning: Setting row names on a tibble is deprecated.

## Warning: Setting row names on a tibble is deprecated.

## Warning: Setting row names on a tibble is deprecated.

## Warning: Setting row names on a tibble is deprecated.

## Warning: Setting row names on a tibble is deprecated.

## Warning: Setting row names on a tibble is deprecated.

## Warning: Setting row names on a tibble is deprecated.

## Warning: Setting row names on a tibble is deprecated.

## Warning: Setting row names on a tibble is deprecated.

knn.fit

## k-Nearest Neighbors 
## 
## 4001 samples
##  784 predictor
##   10 classes: '0', '1', '2', '3', '4', '5', '6', '7', '8', '9' 
## 
## No pre-processing
## Resampling: Cross-Validated (3 fold) 
## Summary of sample sizes: 2667, 2668, 2667 
## Resampling results across tuning parameters:
## 
##   k  Accuracy   Kappa    
##   1  0.9305179  0.9227232
##   2  0.9162703  0.9068747
##   3  0.9240183  0.9154920
##   4  0.9212706  0.9124318
##   5  0.9200192  0.9110354
## 
## Accuracy was used to select the optimal model using the largest value.
## The final value used for the model was k = 1.

plot(knn.fit)

knnPredict <- predict(knn.fit, newdata = X_test) 
confusionMatrix(knnPredict, y_test)

## Confusion Matrix and Statistics
## 
##           Reference
## Prediction   0   1   2   3   4   5   6   7   8   9
##          0 103   0   0   0   0   0   0   0   1   2
##          1   0  99   2   1   2   1   1   1   2   1
##          2   0   3  92   1   0   0   1   0   0   0
##          3   0   0   1  85   0   2   0   0   3   0
##          4   0   0   0   0 108   0   0   1   0   5
##          5   0   0   0   3   0  80   0   0   3   1
##          6   2   0   0   1   1   2 102   0   1   1
##          7   0   0   4   2   0   0   0 102   0   3
##          8   0   0   0   0   0   0   0   0  76   0
##          9   0   0   0   0   1   0   0   4   0  92
## 
## Overall Statistics
##                                           
##                Accuracy : 0.9399          
##                  95% CI : (0.9234, 0.9539)
##     No Information Rate : 0.1121          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.9332          
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: 0 Class: 1 Class: 2 Class: 3 Class: 4 Class: 5
## Sensitivity            0.9810   0.9706  0.92929  0.91398   0.9643  0.94118
## Specificity            0.9966   0.9877  0.99444  0.99338   0.9932  0.99234
## Pos Pred Value         0.9717   0.9000  0.94845  0.93407   0.9474  0.91954
## Neg Pred Value         0.9978   0.9966  0.99224  0.99119   0.9955  0.99452
## Prevalence             0.1051   0.1021  0.09910  0.09309   0.1121  0.08509
## Detection Rate         0.1031   0.0991  0.09209  0.08509   0.1081  0.08008
## Detection Prevalence   0.1061   0.1101  0.09710  0.09109   0.1141  0.08709
## Balanced Accuracy      0.9888   0.9792  0.96187  0.95368   0.9788  0.96676
##                      Class: 6 Class: 7 Class: 8 Class: 9
## Sensitivity            0.9808   0.9444  0.88372  0.87619
## Specificity            0.9911   0.9899  1.00000  0.99441
## Pos Pred Value         0.9273   0.9189  1.00000  0.94845
## Neg Pred Value         0.9978   0.9932  0.98917  0.98559
## Prevalence             0.1041   0.1081  0.08609  0.10511
## Detection Rate         0.1021   0.1021  0.07608  0.09209
## Detection Prevalence   0.1101   0.1111  0.07608  0.09710
## Balanced Accuracy      0.9859   0.9672  0.94186  0.93530

Multinomial Naive Bayes

nb <- multinomial_naive_bayes(select(mnist_raw, -X1), as.factor(mnist_raw$X1), laplace=5)

## Warning: multinomial_naive_bayes(): x was coerced to matrix.

summary(nb)

## 
## ============================ Multinomial Naive Bayes ============================ 
##  
## - Call: multinomial_naive_bayes(x = select(mnist_raw, -X1), y = as.factor(mnist_raw$X1),      laplace = 5) 
## - Laplace: 5 
## - Classes: 10 
## - Samples: 60000 
## - Features: 784 
## - Prior probabilities: 
##     - 0: 0.0987
##     - 1: 0.1124
##     - 2: 0.0993
##     - 3: 0.1022
##     - 4: 0.0974
##     - 5: 0.0904
##     - 6: 0.0986
##     - 7: 0.1044
##     - 8: 0.0975
##     - 9: 0.0992
## 
## ---------------------------------------------------------------------------------

nb_pred <- predict(nb, newdata = data.matrix(select(mnist_raw_test, -X1)), type = "class")

confusionMatrix(nb_pred, as.factor(mnist_raw_test$X1))

## Confusion Matrix and Statistics
## 
##           Reference
## Prediction    0    1    2    3    4    5    6    7    8    9
##          0  912    0   15    4    2   23   18    1    6    6
##          1    0 1061   11   11    2   12   13   21   26    7
##          2    2    5  858   34    6    6   17   11   13    3
##          3    6    9   24  851    0  107    1    5   54   10
##          4    1    0   10    1  732   18    7   19   14   66
##          5    8    2    3   21    0  589   25    0   27    9
##          6   14    6   33    7   25   17  859    1    8    0
##          7    1    0   11   14    1    6    0  861    9   17
##          8   36   51   66   40   38   78   18   40  777   28
##          9    0    1    1   27  176   36    0   69   40  863
## 
## Overall Statistics
##                                           
##                Accuracy : 0.8363          
##                  95% CI : (0.8289, 0.8435)
##     No Information Rate : 0.1135          
##     P-Value [Acc > NIR] : < 2.2e-16       
##                                           
##                   Kappa : 0.818           
##                                           
##  Mcnemar's Test P-Value : NA              
## 
## Statistics by Class:
## 
##                      Class: 0 Class: 1 Class: 2 Class: 3 Class: 4 Class: 5
## Sensitivity            0.9306   0.9348   0.8314   0.8426   0.7454   0.6603
## Specificity            0.9917   0.9884   0.9892   0.9760   0.9849   0.9896
## Pos Pred Value         0.9240   0.9115   0.8984   0.7976   0.8433   0.8611
## Neg Pred Value         0.9925   0.9916   0.9808   0.9822   0.9726   0.9675
## Prevalence             0.0980   0.1135   0.1032   0.1010   0.0982   0.0892
## Detection Rate         0.0912   0.1061   0.0858   0.0851   0.0732   0.0589
## Detection Prevalence   0.0987   0.1164   0.0955   0.1067   0.0868   0.0684
## Balanced Accuracy      0.9611   0.9616   0.9103   0.9093   0.8652   0.8249
##                      Class: 6 Class: 7 Class: 8 Class: 9
## Sensitivity            0.8967   0.8375   0.7977   0.8553
## Specificity            0.9877   0.9934   0.9562   0.9611
## Pos Pred Value         0.8856   0.9359   0.6630   0.7115
## Neg Pred Value         0.9890   0.9816   0.9777   0.9834
## Prevalence             0.0958   0.1028   0.0974   0.1009
## Detection Rate         0.0859   0.0861   0.0777   0.0863
## Detection Prevalence   0.0970   0.0920   0.1172   0.1213
## Balanced Accuracy      0.9422   0.9155   0.8770   0.9082

Support Vector Machine

# svm_base_m <- tune(svm, X1 ~., data = mnist_raw, ranges=list(
#   kernel=c("linear", "polynomial", "radial", "sigmoid"),
#   cost=2^(1:8),
#   epsilon = seq(0,1,0.1)))



#svm_m_best <- svm_base_m$best.model

# mnist_col_reduced <- mnist_raw[ ,colSums(mnist_raw) > 0]
# 
# svm_m_best <- svm(as.factor(mnist_col_reduced$X1) ~., data = mnist_col_reduced, kernel = "linear", cost = 10, scale = FALSE)
# 
# svm_base_pred <- predict(svm_m_best, newdata = adhd_base_testing, type="class")
# svm_base_cm <- confusionMatrix(svm_base_pred, adhd_base_testing$Suicide)
# base_method <- cbind(Reduction_Method = "Base Data", svm_base_m$best.parameters, Training_Accuracy = 1-svm_base_m$best.performance, Testing_Accuracy = sum(diag(svm_base_cm$table))/sum(svm_base_cm$table),
#                      Features=ncol(adhd_base_training)-1,
#                       Cumulative_Proportion=100)
# base_method

##Random Forest

library(randomForest)

## randomForest 4.6-14

## Type rfNews() to see new features/changes/bug fixes.

## 
## Attaching package: 'randomForest'

## The following object is masked from 'package:dplyr':
## 
##     combine

## The following object is masked from 'package:ggplot2':
## 
##     margin

rf <- randomForest(x = X_train , y = y_train, xtest = X_test, ntree = 200)
rf

## 
## Call:
##  randomForest(x = X_train, y = y_train, xtest = X_test, ntree = 200) 
##                Type of random forest: classification
##                      Number of trees: 200
## No. of variables tried at each split: 28
## 
##         OOB estimate of  error rate: 6.47%
## Confusion matrix:
##     0   1   2   3   4   5   6   7   8   9 class.error
## 0 365   0   0   1   0   0   3   2   3   0  0.02406417
## 1   0 453   1   0   2   2   0   2   1   0  0.01735358
## 2   4   2 357   7   5   0   2   6   5   1  0.08226221
## 3   0   2   7 362   2  12   2   5   6   2  0.09500000
## 4   1   2   4   1 393   1   5   2   1  13  0.07092199
## 5   1   2   1   6   3 320   6   1   4   5  0.08309456
## 6   3   2   2   0   2   4 382   1   1   0  0.03778338
## 7   1   6   2   1   6   0   0 419   0   7  0.05203620
## 8   0   8   3   8   5   5   3   0 337   7  0.10372340
## 9   2   1   3   5   6   1   2  13   3 354  0.09230769

plot(rf)

Overall accuracy

1 - mean(rf$err.rate)

## [1] 0.9094193

varImpPlot(rf)